# -*- coding:utf-8 -*-

import os
import math
import time
import thread
import shutil
import commands
import unittest
import threading

from MyException import Error, NoPasswdError, GetSizeError, MoveFileError
from MyLogger import MyLogger, SIZEROTATE, TIMEROTATE
from StatLog import StatLog

class Unzip(threading.Thread):
    def __init__(self, profile, fsinfo, zipfile_list):
        """
        初始化
        """
        #初始化线程类
        threading.Thread.__init__(self)
        self.profile = profile
        self.fsinfo = fsinfo
#        #初始化工作目录
#        self.unzip_src_dir = os.path.join(profile.unzip_dir, "src")
#        self.unzip_tmp_dir = os.path.join(profile.unzip_dir, "tmp")
#        self.work_dir = profile.work_dir
#        #初始化文件缓存目录
#        self.file_cache_dir = profile.file_cache_dir
#        #初始化文件备份目录
#        self.file_backup_dir = profile.file_backup_dir
#        #初始化文件系统信息
#        self.fsinfo = fsinfo
#        #初始化内存盘最大使用率
#        self.mem_max_used = profile.mem_max_used
#        #初始化压缩文件保留开关
#        self.is_save_zipfile = profile.is_save_zipfile
#        #初始化密码列表
#        self.zipfile_passwd_list = profile.zipfile_passwd_list
#        #初始化解压线程数
#        self.unzip_thread_num = profile.unzip_thread_num
        #初始化文件过滤条件
        self.file_filter = ""
        if self.profile.file_type_list != ["all"]: 
            self.file_filter = " *.zip.zip"
            for file_type in self.profile.file_type_list:
                self.file_filter += " *" + file_type
        #初始化压缩文件列表
        self.zipfile_list = zipfile_list
        #初始化解压线程列表
        self.extract_thread_list = []
#        #初始化队列控制信息
#        self.max_zipfile_num = profile.max_zipfile_num
#        self.max_zipnode_num = profile.max_zipnode_num
#        self.max_zipover_num = profile.max_zipover_num
        #初始化日志
        self.success_logger = MyLogger(os.path.join(profile.log_dir, "Unzip_success.log"), SIZEROTATE, profile.size_rotate_backup_num)
        self.error_logger = MyLogger(os.path.join(profile.log_dir, "Unzip_error.log"), SIZEROTATE, profile.size_rotate_backup_num)
        self.status_logger = MyLogger(os.path.join(profile.log_dir, "Unzip_status.log"), SIZEROTATE, profile.size_rotate_backup_num)
        #初始化解压文件数列表
        self.over_stock_list = []
        for i in range(self.profile.max_zip_node_num):
            self.over_stock_list.append(0)
        #初始化统计日志
        self.statlog = None
        if profile.is_write_statlog:
            self.statlog = StatLog(profile.statlog_dir, True)
            self.statlog.daemon = True
            self.statlog.start()
            
    def check_extract_thread(self):
        """
        检查解压线程是否正常
        @return: True-都正常， False-至少有一个不正常
        """
        for extract in self.extract_thread_list:
            if not extract.is_alive():
                return False
        return True
        
    def run(self):
        #清理临时目录      
        self.clear_temp_dir()
        
        #初始化解压线程
        self.init_extract_thread()
            
        #获取压缩文件
        while(True):
            try:
                #检查内存盘使用率，判断是否需要加锁
                self.check_mem_used()
                #检查文件积压数，判断是否需要缓存文件
                self.check_zipfile_list()
            except:
                self.status_logger.exception("unknown error occur in unzip monitor thread")
                raise

            time.sleep(20)
            
    def check_zipfile_list(self):
        """
        检查压缩文件列表，如果压缩文件列表中积压文件数过大，则需要缓存文件
        """
        #获取压缩文件列表积压文件数
        over_stock_num = self.zipfile_list.size()   
        self.over_stock_list.pop(0)
        self.over_stock_list.append(over_stock_num)
        #print ", ".join([str(over_stock) for over_stock in self.over_stock_list])
        #判断是否需要缓存文件
        #print over_stock_num, self.profile.max_zip_file_num
        if over_stock_num > self.profile.max_zip_file_num:
            #计算历史积压文件数超过最大值的次数
            over_stock_count = 0
            for over_stock in self.over_stock_list:
                if over_stock > self.profile.max_zip_file_num:
                    over_stock_count += 1
            #需要缓存压缩文件
            #print over_stock_count, self.profile.max_zip_over_num
            if over_stock_count > self.profile.max_zip_over_num:
                self.status_logger.info("Zip file list block. over stock: %s"%(", ".join([str(over_stock) for over_stock in self.over_stock_list])))
                zipfile_cache_dir = os.path.join(self.profile.file_cache_dir, time.strftime("%Y%m%d%H", time.localtime()))
                if not os.path.exists(zipfile_cache_dir):
                    os.makedirs(zipfile_cache_dir)
                #print self.zipfile_list.size(), self.profile.max_zip_file_num
                while self.zipfile_list.size() > self.profile.max_zip_file_num:
                    filename = self.zipfile_list.get_force()
                    src_file = os.path.join(self.profile.unzip_dir, "src", filename)
                    dst_file = os.path.join(zipfile_cache_dir, filename)
                    shutil.move(src_file, dst_file)
                    self.status_logger.info("Cache zip file %s"%(filename))
                    
    def check_mem_used(self):
        """
        检查内存盘使用率，如果内存盘使用率过大，需要给压缩文件列表加锁停止解压；如果内存盘使用率正常，需要给文件列表解锁
        """
        mem_used = self.fsinfo.mem_used
        if mem_used > self.profile.mem_max_used:
            list_flag = self.zipfile_list.get_flag()
            if list_flag:
                self.zipfile_list.set_flag(False)
                self.status_logger.info("Lock zip file list. mem used: %d%%"%(mem_used))
        else:
            list_flag = self.zipfile_list.get_flag()
            if not list_flag:
                self.zipfile_list.set_flag(True)
                self.status_logger.info("Unlock zip file list. mem used: %d%%"%(mem_used))
                    
    def clear_temp_dir(self):
        """
        清理临时目录中上次未解压的文件
        """
        unzip_tmp_dir = os.path.join(self.profile.unzip_dir, "tmp")
        if os.path.exists(unzip_tmp_dir):
            shutil.rmtree(unzip_tmp_dir)
        os.makedirs(unzip_tmp_dir)
        
    def init_extract_thread(self):
        """
        初始化解压线程
        """
        unzip_thread_num = self.profile.unzip_thread_num
        if unzip_thread_num < 1:
            unzip_thread_num = 1
        if unzip_thread_num > 64:
            unzip_thread_num = 64
        for i in range(unzip_thread_num):
            extract = Extract(self.profile, self.zipfile_list, self.file_filter, self.success_logger, self.error_logger, self.statlog)
            extract.setDaemon(True)
            extract.start()
            self.extract_thread_list.append(extract)
    
class Extract(threading.Thread):
    def __init__(self, profile, zipfile_list, file_filter, success_logger, error_logger, statlog):
        """
        初始化
        """
        #初始化线程类
        threading.Thread.__init__(self)
        self.profile = profile
#        #初始化工作目录
#        self.unzip_src_dir = unzipSrcDir
#        self.unzip_tmp_dir = unzipTmpDir
#        self.work_dir = workDir
#        #初始化错误文件备份目录
#        self.file_backup_dir = file_backup_dir
#        #初始化保留压缩文件开关
#        self.is_save_zipfile = is_save_zipFile
        #初始化压缩文件列表
        self.zipfile_list = zipfile_list
#        #初始化压缩文件密码列表
#        self.zipfile_passwd_list = zipfile_passwd_list
        #初始化过滤条件
        self.file_filter = file_filter
        #初始化日志
        self.success_logger = success_logger
        self.error_logger = error_logger
        self.statlog = statlog
        
    def make_temp_dir(self):
        """
        创建临时工作目录
        """
        self.unzip_tmp_dir = os.path.join(self.profile.unzip_dir, "tmp", str(int(math.fabs(thread.get_ident()))))
        if not os.path.exists(self.unzip_tmp_dir):
            os.makedirs(self.unzip_tmp_dir)
            
    def unzip_file(self, zipfile):
        """
        解压压缩文件
        @param zipfile: 压缩文件路径
        """
        strCmd = "unzip -o -P 116611 " + zipfile + " -d " + self.unzip_tmp_dir + self.file_filter
        #print strCmd
        state, output = commands.getstatusoutput(strCmd)
        if output.find("incorrect password") > -1:
            state = -1
                    
        if state == 0 or state == 256 or state == 2816:
            tmpfile = os.path.join(self.unzip_tmp_dir, os.path.basename(zipfile) + '.zip')
            if os.path.exists(tmpfile):
                strCmd = "unzip -o " + tmpfile + " -d " + self.unzip_tmp_dir + self.file_filter
                #print strCmd
                state, output = commands.getstatusoutput(strCmd)
                #print state, output
                os.remove(tmpfile)
        else:
            for password in self.profile.zipfile_passwd_list:
                strCmd = "unzip -o -P " + password + " " + zipfile + " -d " + self.unzip_tmp_dir + self.file_filter
                #print strCmd
                state, output = commands.getstatusoutput(strCmd)
                if state == 0 or state == 256 or state == 2816:
                    if output.find("incorrect password") == -1:
                        break
                    else:
                        state = -1
                    
        if state != 0 and state != 256 and state != 2816:
            raise NoPasswdError("no passwd for zip file: %s"%(os.path.basename(zipfile)))
        
    def run(self):
        #创建解压临时目录
        self.make_temp_dir()
        
        #解压文件
        while(True):
            try:
                try:
                    #获取压缩文件
                    filename = self.zipfile_list.get()
                    if not filename:
                        time.sleep(3)
                        continue
                        
                    #拼接文件路径
                    zipfile = os.path.join(self.profile.unzip_dir, "src", filename)
                    filesize = 0
                        
                    #获取压缩文件大小
                    try:
                        filesize = os.path.getsize(zipfile)
                    except os.error, msg:
                        raise GetSizeError(msg)
                        
                    #解压文件到临时目录
                    try:
                        self.unzip_file(zipfile)
                    except NoPasswdError, msg:
                        shutil.rmtree(self.unzip_tmp_dir)
                        os.makedirs(self.unzip_tmp_dir)
                        backup_dir = os.path.join(self.profile.file_backup_dir, "Error_Zip", time.strftime("%Y%m%d%H", time.localtime()))
                        if not os.path.exists(backup_dir):
                            os.makedirs(backup_dir)
                        dst_file = os.path.join(backup_dir, filename)
                        shutil.move(zipfile, dst_file)
                        raise
                        
                    #移动解压文件到工作目录
                    try:   
                        #移动文件到接受目录
                        file_list = os.listdir(self.unzip_tmp_dir)
                        for file in file_list:
                            src_file = os.path.join(self.unzip_tmp_dir, file)
                            dst_file = os.path.join(self.profile.work_dir, file)
                            if os.path.isfile(src_file):
                                shutil.move(src_file, dst_file)
                            else:      
                                shutil.rmtree(src_file)
                    except Exception, msg:
                        shutil.rmtree(self.unzip_tmp_dir)
                        os.makedirs(self.unzip_tmp_dir)
                        raise MoveFileError(msg)
                        
                    #保留压缩文件
                    if self.profile.is_save_zipfile:
                        save_file = os.path.join(self.profile.work_dir, filename)
                        shutil.move(zipfile, save_file)
                    else:
                        os.remove(zipfile)
                        
                    self.success_logger.info("Success unzip file: %s size: %d"%(filename, filesize))
                    if self.profile.is_write_statlog:
                        self.statlog.write_statlog_curtly(filename, filesize, "zip", 0, "unzip", "")
                except GetSizeError, msg:
                    self.error_logger.exception("Error unzip file: %s size: %d cause: GetSize"%(filename, filesize))
                    if self.profile.is_write_statlog:
                        self.statlog.write_statlog_curtly(filename, filesize, "zip", -1, "unzip", "")
                except NoPasswdError, msg:
                    self.error_logger.exception("Error unzip file: %s size: %d cause: NoPasswd"%(filename, filesize))
                    if self.profile.is_write_statlog:
                        self.statlog.write_statlog_curtly(filename, filesize, "zip", -1, "unzip", "")
                except MoveFileError, msg:
                    self.error_logger.exception("Error unzip file: %s size: %d cause: MoveFile"%(filename, filesize))
                    if self.profile.is_write_statlog:
                        self.statlog.write_statlog_curtly(filename, filesize, "zip", -1, "unzip", "")
                except Exception, msg:
                    self.error_logger.exception("Error unzip file: %s size: %d cause: Unknown"%(filename, filesize))
                    if self.profile.is_write_statlog:
                        self.statlog.write_statlog_curtly(filename, filesize, "zip", -1, "unzip", "")
            except:
                self.error_logger.exception("unknown error")
                     
import unittest
from ConfigReader import ConfigReader
from ConfigAssist import ConfigAssist
from MutexList import MutexList
from FSInfo import FSInfo              
       
class UnzipTestCase(unittest.TestCase):
    def setUp(self):
        profile = ConfigReader().read_profile()

        alL_app_conf, result = ConfigAssist().check_match_rule()
        
        self.zipfile_list = MutexList()
        fsinfo = FSInfo()
        
        if not os.path.exists(profile.receive_dir):
            os.makedirs(profile.receive_dir)
        if not os.path.exists(profile.receive_tmp_dir):
            os.makedirs(profile.receive_tmp_dir)
        if not os.path.exists(profile.work_dir):
            os.makedirs(profile.work_dir)
        if not os.path.exists(profile.unzip_dir):
            os.makedirs(profile.unzip_dir)
        if not os.path.exists(profile.file_cache_dir):
            os.makedirs(profile.file_cache_dir)
        if not os.path.exists(profile.file_backup_dir):
            os.makedirs(profile.file_backup_dir)
        
        self.src_dir = os.path.join(profile.unzip_dir, "src")
        if not os.path.exists(self.src_dir):
            os.makedirs(self.src_dir)
        
        unzip_success = logging.getLogger("receive_success")
        unzip_error = logging.getLogger("receive_error")
        unzip_status = logging.getLogger("receive_cache")
        
        self.unzip_thread = Unzip(profile, fsinfo, self.zipfile_list, unzip_success, unzip_error, unzip_status)
        
    def tearDown(self):
        pass
    
    def test_base_func(self):
        self.unzip_thread.daemon = True
        self.unzip_thread.start()
        
        while self.unzip_thread.is_alive():
            file_list = os.listdir(self.src_dir)
            for filename in file_list:
                self.zipfile_list.add(filename)
            time.sleep(10)
        print "exit"
            
if __name__ == "__main__":
    suite = unittest.TestSuite()
    suite.addTest(UnzipTestCase("test_base_func"))
    
    runner = unittest.TextTestRunner()
    runner.run(suite)
    
            